#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Author: 邵奈一
@Email: shaonaiyi@163.com
@Date: 2024/11/18
@微信：shaonaiyi888
@微信公众号: 邵奈一 
"""
# 代码5-1
# 数据处理
import numpy as np


class Preprocessing:

    @staticmethod
    def read_dataset(file):

        letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
                   'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', ' ']
        # 打开原始文件
        with open(file, 'r') as f:
            raw_text = f.readlines()

        # 将每一行转换为小写
        raw_text = [line.lower() for line in raw_text]

        # 创建一个包含整个文本的字符串
        text_string = ''
        for line in raw_text:
            text_string += line.strip()

        # 创建一个字符数组
        text = list()
        for char in text_string:
            text.append(char)

        # 去掉所有的符号，只保留字母
        text = [char for char in text if char in letters]

        print(f"Text length after preprocessing: {len(text)}")
        return text

    # 代码5-2
    # 字典创建
    @staticmethod
    def create_dictionary(text):

        char_to_idx = dict()
        idx_to_char = dict()

        idx = 0
        for char in text:
            if char not in char_to_idx.keys():
                # 构建字典
                char_to_idx[char] = idx
                idx_to_char[idx] = char
                idx += 1

        print(f'Vocab size: {len(char_to_idx)}')
        return char_to_idx, idx_to_char

    # 代码5-3
    # 序列生成
    @staticmethod
    def build_sequences_target(text, char_to_idx, window):
        x = list()
        y = list()

        for i in range(len(text)):
            try:
                # 从文本中获取字符窗口
                # 将其转换为其idx表示
                sequence = text[i: i + window]
                sequence = [char_to_idx[char] for char in sequence]

                # 得到target
                # 转换到它的idx表示
                target = text[i + window]
                target = char_to_idx[target]

                # 保存sequence和target
                x.append(sequence)
                y.append(target)
            except:
                pass

        x = np.array(x)
        y = np.array(y)

        print(f'Shape of sequences (X): {x.shape}')
        print(f'Shape of targets (Y): {y.shape}')
        return x, y